# Updated by Wei on 20140823
# useful

from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

def compute_gainRate_for_each_query():
    # fallThroughRate_weight_0_OR_Debug
    # fallThroughRate_weight_0_OR_1%_gainRateAdded
    # fallThroughRate_weight_0_OR_5%_gainRateAdded
    # fallThroughRate_weight_0_OR_15%_gainRateAdded
    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/fallThroughRate_weight_150_OR_Debug_gainRateAdded"
    outputFileHandler = open(outputFileName,"w")
    
    # fallThroughRate_weight_0_OR_5%
    # inputFileName = "/data/obukai/workspace_USE_SINCE_20140217Night/web-search-engine-wei-2014-March/fallThroughRate_weight_0_OR_Debug"
    inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/fallThroughRate_weight_150_OR_Debug"
    inputFileHandler = open(inputFileName,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentNumOfDocumentResultsReturnedForFirstTier = int(lineElements[1])
        NumOfDocumentResultsReturnedInTotal = int(lineElements[2])
        cost = int(lineElements[4])
        gain = NumOfDocumentResultsReturnedInTotal - currentNumOfDocumentResultsReturnedForFirstTier
        gainRate = gain / cost
        outputFileHandler.write(line.strip() + " " + str(gainRate) + "\n")
    print "Overall:"
    print "inputFileName:",inputFileName
    print "outputFileName:",outputFileName
    inputFileHandler.close()
    outputFileHandler.close()

def compute_sth():
    inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/fallThroughRate_weight_150_OR_Debug_gainRateAdded"
    inputFileHandler = open(inputFileName,"r")
    numOfRelatedDocumentResultReturned = 0
    numOfDocumentResultInTotal = 0
    totalQueryEvaluationCostForCPUFromFirstTier = 0
    totalQueryEvaluationCostForCPUFromSecondTier = 0
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        numOfRelatedDocumentResultReturned += int(lineElements[1])
        numOfDocumentResultInTotal += int(lineElements[2])
        totalQueryEvaluationCostForCPUFromFirstTier += int(lineElements[3])
        totalQueryEvaluationCostForCPUFromSecondTier += int(lineElements[4])
    print "numOfDocumentResultInTotal:",numOfDocumentResultInTotal
    print "numOfRelatedDocumentResultReturned:",numOfRelatedDocumentResultReturned
    print "totalQueryEvaluationCostForCPUFromFirstTier:",totalQueryEvaluationCostForCPUFromFirstTier
    print "totalQueryEvaluationCostForCPUFromSecondTier:",totalQueryEvaluationCostForCPUFromSecondTier
    inputFileHandler.close()

def compute_overlap_cost_measure():
    # the question is: do you want to go to the second tier, what is the trade-off about this
    # use whatever method you have to do it
    
    # percentage level needed to investigate
    # [0.01,0.03,0.05,0.1,0.15,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]
    TOTAL_NUM_OF_QUERIES = 4981
    numOfQueriesBackSearchAt1Percentage = int(TOTAL_NUM_OF_QUERIES * 0.01)
    numOfQueriesBackSearchAt3Percentage = int(TOTAL_NUM_OF_QUERIES * 0.03)
    numOfQueriesBackSearchAt5Percentage = int(TOTAL_NUM_OF_QUERIES * 0.05)
    numOfQueriesBackSearchAt10Percentage = int(TOTAL_NUM_OF_QUERIES * 0.1)
    numOfQueriesBackSearchAt15Percentage = int(TOTAL_NUM_OF_QUERIES * 0.15)
    numOfQueriesBackSearchAt20Percentage = int(TOTAL_NUM_OF_QUERIES * 0.2)
    numOfQueriesBackSearchAt30Percentage = int(TOTAL_NUM_OF_QUERIES * 0.3)
    numOfQueriesBackSearchAt40Percentage = int(TOTAL_NUM_OF_QUERIES * 0.4)
    numOfQueriesBackSearchAt50Percentage = int(TOTAL_NUM_OF_QUERIES * 0.5)
    numOfQueriesBackSearchAt60Percentage = int(TOTAL_NUM_OF_QUERIES * 0.6)
    numOfQueriesBackSearchAt70Percentage = int(TOTAL_NUM_OF_QUERIES * 0.7)
    numOfQueriesBackSearchAt80Percentage = int(TOTAL_NUM_OF_QUERIES * 0.8)
    numOfQueriesBackSearchAt90Percentage = int(TOTAL_NUM_OF_QUERIES * 0.9)
    print "numOfQueriesBackSearchAt1Percentage:",numOfQueriesBackSearchAt1Percentage
    print "numOfQueriesBackSearchAt3Percentage:",numOfQueriesBackSearchAt3Percentage
    print "numOfQueriesBackSearchAt5Percentage:",numOfQueriesBackSearchAt5Percentage
    print "numOfQueriesBackSearchAt10Percentage:",numOfQueriesBackSearchAt10Percentage
    print "numOfQueriesBackSearchAt15Percentage:",numOfQueriesBackSearchAt15Percentage
    print "numOfQueriesBackSearchAt20Percentage:",numOfQueriesBackSearchAt20Percentage
    print "numOfQueriesBackSearchAt30Percentage:",numOfQueriesBackSearchAt30Percentage
    print "numOfQueriesBackSearchAt40Percentage:",numOfQueriesBackSearchAt40Percentage
    print "numOfQueriesBackSearchAt50Percentage:",numOfQueriesBackSearchAt50Percentage
    print "numOfQueriesBackSearchAt60Percentage:",numOfQueriesBackSearchAt60Percentage
    print "numOfQueriesBackSearchAt70Percentage:",numOfQueriesBackSearchAt70Percentage
    print "numOfQueriesBackSearchAt80Percentage:",numOfQueriesBackSearchAt80Percentage
    print "numOfQueriesBackSearchAt90Percentage:",numOfQueriesBackSearchAt90Percentage
    numOfQueriesBackSearchDict = {}
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt1Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt3Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt5Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt10Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt15Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt20Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt30Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt40Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt50Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt60Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt70Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt80Percentage] = 1
    numOfQueriesBackSearchDict[numOfQueriesBackSearchAt90Percentage] = 1
    
    queryList = []
    # fallThroughRate_weight_0_OR_1%_gainRateAdded_sortedByGainRate
    # fallThroughRate_weight_0_OR_5%_gainRateAdded_sortedByGainRate
    # fallThroughRate_weight_0_OR_15%_gainRateAdded_sortedByGainRate
    inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/fallThroughRate_weight_150_OR_Debug_gainRateAdded"
    inputFileHandler = open(inputFileName,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentQueryID = lineElements[0]
        numOfResultsInFirstTier = int(lineElements[1])
        numOfResultsInTotal = int(lineElements[2])
        costInFirstTier = int( lineElements[3] )
        costInSecondTier = int( lineElements[4] )
        gainRate = float( lineElements[5] )
        queryTuple = (currentQueryID,numOfResultsInFirstTier,numOfResultsInTotal,costInFirstTier,costInSecondTier,gainRate)
        queryList.append(queryTuple)
    
    print "len(queryList):",len(queryList)
    queryList.sort(cmp=None, key=itemgetter(4), reverse=False)
    
    
    # 0.08% & weight == 0
    NUM_OF_DOCUMENT_RESULT_IN_TOTAL = 49808
    numOfRelatedDocumentResultReturnedOnlyUsingFirstTier = 2599
    totalQueryEvaluationCostForCPUFromFirstTier  = 152834089
    totalQueryEvaluationCostForCPUFromSecondTier = 70108204014    
    
    # 0.08% & weight == 0
    # NUM_OF_DOCUMENT_RESULT_IN_TOTAL = 49808
    # numOfRelatedDocumentResultReturnedOnlyUsingFirstTier = 2162
    # totalQueryEvaluationCostForCPUFromFirstTier  = 121577204
    # totalQueryEvaluationCostForCPUFromSecondTier = 70108204014    
    
    # 1%
    # NUM_OF_DOCUMENT_RESULT_IN_TOTAL = 49808
    # numOfRelatedDocumentResultReturnedOnlyUsingFirstTier = 8226
    # totalQueryEvaluationCostForCPUFromFirstTier  = 6510423080
    # totalQueryEvaluationCostForCPUFromSecondTier = 70108204014
        
    # 5%
    # NUM_OF_DOCUMENT_RESULT_IN_TOTAL = 49808
    # numOfRelatedDocumentResultReturnedOnlyUsingFirstTier = 18125
    # totalQueryEvaluationCostForCPUFromFirstTier  = 23171262303
    # totalQueryEvaluationCostForCPUFromSecondTier = 70108204014
    
    # 15%
    # NUM_OF_DOCUMENT_RESULT_IN_TOTAL = 49808
    # numOfRelatedDocumentResultReturnedOnlyUsingFirstTier = 28248
    # totalQueryEvaluationCostForCPUFromFirstTier  = 49950523898
    # totalQueryEvaluationCostForCPUFromSecondTier = 70108204014
        
    print "0",numOfRelatedDocumentResultReturnedOnlyUsingFirstTier,totalQueryEvaluationCostForCPUFromFirstTier,numOfRelatedDocumentResultReturnedOnlyUsingFirstTier/NUM_OF_DOCUMENT_RESULT_IN_TOTAL
    for i in range(0,len(queryList)):
        (currentQueryID,numOfResultsInFirstTier,numOfResultsInTotal,costInFirstTier,costInSecondTier,gainRate) = queryList[i]
        numOfRelatedDocumentResultReturnedOnlyUsingFirstTier += numOfResultsInTotal - numOfResultsInFirstTier
        totalQueryEvaluationCostForCPUFromFirstTier += costInSecondTier
        if i in numOfQueriesBackSearchDict:
            print str(i),numOfRelatedDocumentResultReturnedOnlyUsingFirstTier,totalQueryEvaluationCostForCPUFromFirstTier,numOfRelatedDocumentResultReturnedOnlyUsingFirstTier/NUM_OF_DOCUMENT_RESULT_IN_TOTAL
    print str(i),numOfRelatedDocumentResultReturnedOnlyUsingFirstTier,totalQueryEvaluationCostForCPUFromFirstTier,numOfRelatedDocumentResultReturnedOnlyUsingFirstTier/NUM_OF_DOCUMENT_RESULT_IN_TOTAL
    inputFileHandler.close()

# step1:
# compute_gainRate_for_each_query()
# step2:
# compute_sth()
# step3:
compute_overlap_cost_measure()






